package com.solt.prism.core;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;

import org.apache.log4j.Logger;
import org.w3c.dom.Document;

import com.solt.prism.core.conf.Configurable;
import com.solt.prism.core.conf.Configuration;
import com.solt.prism.core.conf.PrismConfiguration;
import com.solt.prism.core.conf.news.Category;
import com.solt.prism.core.conf.news.NewsConf;
import com.solt.prism.core.entity.News;
import com.solt.prism.core.utils.NewsPersistencer;
import com.solt.prism.core.utils.PrismConstants;
import com.solt.prism.plugins.NewsCrawler;
import com.solt.prism.plugins.NewsWrapper;
import com.solt.prism.plugins.PluginException;

public abstract class AbstractCrawler implements NewsCrawler, Configurable {
	private static final Logger logger = Logger
			.getLogger(AbstractCrawler.class);
	private static final boolean debugMode = PrismConfiguration.getInstance()
			.getBoolean(PrismConstants.CRAWL_DEBUG_MODE, false);
	private static final boolean useCrawlTime = PrismConfiguration
			.getInstance().getBoolean(
					PrismConstants.NEWS_PUBDATE_USING_CRAWLTIME, false);
	private Iterator<Entry<URL, Integer>> urlCatsIter;
	private Map<URL, Integer> urlCats;
	protected PageFetcher pageFetcher;
	protected volatile boolean shutdowned;
	protected Configuration conf;
	protected NewsConf newsConf;
	protected URL homeSite;
	protected Date lastAccessTime;
	protected Date nextAccessTime;
	private DateFormat formater = DateFormat.getDateTimeInstance();
	private String lastTimeFile;
	private NewsWrapper detailWrapper;
	private NewsPersistencer persistencer;
	private long sleepMillisTime;

	public AbstractCrawler() {
		pageFetcher = new PageFetcher();
	}

	@Override
	public void init(NewsConf newsConf, NewsWrapper wrapper,
			NewsPersistencer persistencer) throws PluginException {
		this.newsConf = newsConf;
		this.detailWrapper = wrapper;
		this.persistencer = persistencer;
		List<Category> cats = newsConf.getCategories().getCategory();
		if (cats.isEmpty() || (urlCats = parseUrlCats(cats)).isEmpty()) {
			shutdown();
			return;
		}

		try {
			URL url = urlCats.keySet().iterator().next();
			homeSite = new URL(url, "/");
			conf.set(PrismConstants.PLUGIN_HOME_SITE, homeSite.toString());
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
		urlCatsIter = urlCats.entrySet().iterator();
		sleepMillisTime = PrismConfiguration.getInstance().getLong(
				PrismConstants.CRAWL_CYCLE_MILLISTIME, -1);
		if (sleepMillisTime < 0) {
			sleepMillisTime = -1;
		}
		lastTimeFile = conf.get(PrismConstants.PLUGIN_DIR) + File.separatorChar
				+ PrismConstants.PLUGIN_ACCESS_TIME_FILE;
		lastAccessTime = getLastAcessTime(lastTimeFile);
		nextAccessTime = new Date(lastAccessTime.getTime());
	}

	private Map<URL, Integer> parseUrlCats(List<Category> cats) {
		Map<URL, Integer> urlCats = new HashMap<URL, Integer>();
		String[] urls = null;
		int catId = 0;
		for (Category cat : cats) {
			urls = cat.getValue().split("\\s+");
			catId = cat.getId();
			for (String url : urls) {
				if (!url.isEmpty()) {
					try {
						urlCats.put(new URL(url), catId);
					} catch (MalformedURLException e) {
						logger.error("malformed url: " + url, e);
					}
				}
			}
		}
		return urlCats;
	}

	/**
	 * @param string
	 * @return
	 */
	private Date getLastAcessTime(String timeFile) {
		BufferedReader reader = null;
		try {
			reader = new BufferedReader(new FileReader(timeFile));
			String timeString = reader.readLine();
			if (timeString != null
					&& !((timeString = timeString.trim()).isEmpty())) {
				return formater.parse(timeString);
			}
		} catch (IOException e) {
		} catch (ParseException e) {
			logger.warn(timeFile + " contain invalid time", e);
		} finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		return new Date(0);
	}

	private void setLastAcessTime(String timeFile) {
		PrintWriter writer = null;
		try {
			writer = new PrintWriter(new FileWriter(timeFile));
			String timeString = formater.format(lastAccessTime);
			writer.println(timeString);
		} catch (IOException e) {
			logger.warn("error when writting time to " + timeFile, e);
		} finally {
			if (writer != null) {
				writer.close();
			}
		}
	}

	@Override
	public void crawl() throws InterruptedException {
		Document htmlDom = null;
		Date createTime = null;
		while (!shutdowned) {
			News news = new News();
			if (!computeCrawlNextNews(news)) {
				htmlDom = pageFetcher.retrieve(news.getSourceUrl());
				if (htmlDom == null
						|| (htmlDom != null && !detailWrapper.extract(htmlDom, news))) {
					continue;
				}
			}
			
			createTime = news.getCreateTime();
			if (createTime == null) {
				logger.warn("can't detect createTime field of news: "
						+ news);
			} else if (!createTime.after(lastAccessTime)) {
				continue;
			} else if (createTime.after(nextAccessTime)) {
				nextAccessTime.setTime(createTime.getTime());
			}
			
			if (useCrawlTime) {
				news.setCreateTime(new Date());
			}
			if (isValid(news) && !debugMode) {
				persistencer.store(news);
			}
		}
	}

	protected boolean isValid(News news) {
		return news.getHtmlContent() != null && news.getTitle() != null && news.getDescription() != null;
	}

	/**
	 * chu yeu phim url source, category bai bao
	 * 
	 * @param news
	 * @return
	 * @throws InterruptedException
	 */
	public boolean computeCrawlNextNews(News news) throws InterruptedException {
		try {
			return makeNewsTask(news, urlCatsIter);
		} catch (NoSuchElementException e) {
			persistencer.sync();
			lastAccessTime.setTime(nextAccessTime.getTime());
			setLastAcessTime(lastTimeFile);
			if (sleepMillisTime != -1) {
				Thread.sleep(sleepMillisTime);
			}
			urlCatsIter = urlCats.entrySet().iterator();
		}
		return false;
	}

	protected abstract boolean makeNewsTask(News news,
			Iterator<Entry<URL, Integer>> urlCatsIter)
			throws NoSuchElementException;

	@Override
	public void shutdown() {
		shutdowned = true;
	}

	@Override
	public boolean isShutdowned() {
		return shutdowned;
	}

	@Override
	public void setConf(Configuration conf) {
		this.conf = conf;
	}

	@Override
	public Configuration getConf() {
		return conf;
	}

	public NewsConf getNewsConf() {
		return newsConf;
	}

	public URL getHomeSite() {
		return homeSite;
	}
}
